Sākums Izpētīt Palīdzība
Reģistrēties Pierakstīties
J1X
/
huginn
Vērot 2
Pievienot zvaigznīti 0
Atdalīts 0

Pārlūkot izejas kodu
Merge branch 'master' into do_not_symbolize

Conflicts:
app/models/agents/human_task_agent.rb
spec/models/agents/human_task_agent_spec.rb

Andrew Cantino 11 gadi atpakaļ
vecāks
21c0e5911d 5461ec3d2f
revīzija
73e8406d4f
6 mainītis faili ar 496 papildinājumiem un 68 dzēšanām
Rādīt salīdzināšanas statistiku
  1. 3 3
      Gemfile.lock
  2. 198 45
      app/models/agents/human_task_agent.rb
  3. 62 0
      app/models/agents/webhook_agent.rb
  4. 30 1
      lib/capistrano/sync.rb
  5. 172 19
      spec/models/agents/human_task_agent_spec.rb
  6. 31 0
      spec/models/agents/webhook_agent_spec.rb

+ 3 - 3
Gemfile.lock
Parādīt failu 

@@ -63,9 +63,9 @@ GEM
63 63 
       railties (>= 3.2.6, < 5)
64 64 
       warden (~> 1.2.3)
65 65 
     diff-lcs (1.2.4)
66 
-    dotenv (0.8.0)
67 
-    dotenv-rails (0.8.0)
68 
-      dotenv (= 0.8.0)
66 
+    dotenv (0.9.0)
67 
+    dotenv-rails (0.9.0)
68 
+      dotenv (= 0.9.0)
69 69 
     em-http-request (1.0.3)
70 70 
       addressable (>= 2.2.3)
71 71 
       cookiejar

+ 198 - 45
app/models/agents/human_task_agent.rb
Parādīt failu 

@@ -9,9 +9,13 @@ module Agents
9 9
10 10 
       HITs can be created in response to events, or on a schedule.  Set `trigger_on` to either `schedule` or `event`.
11 11
12 
+      # Schedule
13 
+
12 14 
       The schedule of this Agent is how often it should check for completed HITs, __NOT__ how often to submit one.  To configure how often a new HIT
13 15 
       should be submitted when in `schedule` mode, set `submission_period` to a number of hours.
14 16
17 
+      # Example
18 
+
15 19 
       If created with an event, all HIT fields can contain interpolated values via [JSONPaths](http://goessner.net/articles/JsonPath/) placed between < and > characters.
16 20 
       For example, if the incoming event was a Twitter event, you could make a HITT to rate its sentiment like this:
17 21 
 
@@ -58,8 +62,52 @@ module Agents
58 62 
       which contain `key` and `text`.  For _free\\_text_, the special configuration options are all optional, and are
59 63 
       `default`, `min_length`, and `max_length`.
60 64
61 
-      If all of the `questions` are of `type` _selection_, you can set `take_majority` to _true_ at the top level to
62 
-      automatically select the majority vote for each question across all `assignments`.  If all selections are numeric, an `average_answer` will also be generated.
65 
+      # Combining answers
66 
+
67 
+      There are a couple of ways to combine HITs that have multiple `assignments`, all of which involve setting `combination_mode` at the top level.
68 
+
69 
+      ## Taking the majority
70 
+
71 
+      Option 1: if all of your `questions` are of `type` _selection_, you can set `combination_mode` to `take_majority`.
72 
+      This will cause the Agent to automatically select the majority vote for each question across all `assignments` and return it as `majority_answer`.
73 
+      If all selections are numeric, an `average_answer` will also be generated.
74 
+
75 
+      Option 2: you can have the Agent ask additional human workers to rank the `assignments` and return the most highly ranked answer.
76 
+      To do this, set `combination_mode` to `poll` and provide a `poll_options` object.  Here is an example:
77 
+
78 
+          {
79 
+            "trigger_on": "schedule",
80 
+            "submission_period": 12,
81 
+            "combination_mode": "poll",
82 
+            "poll_options": {
83 
+              "title": "Take a poll about some jokes",
84 
+              "instructions": "Please rank these jokes from most funny (5) to least funny (1)",
85 
+              "assignments": 3,
86 
+              "row_template": "<$.joke>"
87 
+            },
88 
+            "hit": {
89 
+              "assignments": 5,
90 
+              "title": "Tell a joke",
91 
+              "description": "Please tell me a joke",
92 
+              "reward": 0.05,
93 
+              "lifetime_in_seconds": "3600",
94 
+              "questions": [
95 
+                {
96 
+                  "type": "free_text",
97 
+                  "key": "joke",
98 
+                  "name": "Your joke",
99 
+                  "required": "true",
100 
+                  "question": "Joke",
101 
+                  "min_length": "2",
102 
+                  "max_length": "2000"
103 
+                }
104 
+              ]
105 
+            }
106 
+          }
107 
+
108 
+      Resulting events will have the original `answers`, as well as the `poll` results, and a field called `best_answer` that contains the best answer as determined by the poll.
109 
+
110 
+      # Other settings
63 111
64 112 
       `lifetime_in_seconds` is the number of seconds a HIT is left on Amazon before it's automatically closed.  The default is 1 day.
65 113 
 
@@ -70,6 +118,12 @@ module Agents
70 118 
       Events look like:
71 119
72 120 
           {
121 
+            "answers": [
122 
+              {
123 
+                "feedback": "Hello!",
124 
+                "sentiment": "happy"
125 
+              }
126 
+            ]
73 127 
           }
74 128 
     MD
75 129 
 
@@ -97,9 +151,13 @@ module Agents
97 151 
         errors.add(:base, "all questions of type 'selection' must have a selections array with selections that set 'key' and 'name'")
98 152 
       end
99 153
100 
-      if options['take_majority'] == "true" && options['hit']['questions'].any? { |question| question['type'] != "selection" }
154 
+      if take_majority? && options['hit']['questions'].any? { |question| question['type'] != "selection" }
101 155 
         errors.add(:base, "all questions must be of type 'selection' to use the 'take_majority' option")
102 156 
       end
157 
+
158 
+      if create_poll?
159 
+        errors.add(:base, "poll_options is required when combination_mode is set to 'poll' and must have the keys 'title', 'instructions', 'row_template', and 'assignments'") unless options['poll_options'].is_a?(Hash) && options['poll_options']['title'].present? && options['poll_options']['instructions'].present? && options['poll_options']['row_template'].present? && options['poll_options']['assignments'].to_i > 0
160 
+      end
103 161 
     end
104 162
105 163 
     def default_options
 
@@ -152,69 +210,151 @@ module Agents
152 210
153 211 
       if options['trigger_on'] == "schedule" && (memory['last_schedule'] || 0) <= Time.now.to_i - options['submission_period'].to_i * 60 * 60
154 212 
         memory['last_schedule'] = Time.now.to_i
155 
-        create_hit
213 
+        create_basic_hit
156 214 
       end
157 215 
     end
158 216
159 217 
     def receive(incoming_events)
160 218 
       if options['trigger_on'] == "event"
161 219 
         incoming_events.each do |event|
162 
-          create_hit event
220 
+          create_basic_hit event
163 221 
         end
164 222 
       end
165 223 
     end
166 224
167 225 
     protected
168 226
227 
+    def take_majority?
228 
+      options['combination_mode'] == "take_majority" || options['take_majority'] == "true"
229 
+    end
230 
+
231 
+    def create_poll?
232 
+      options['combination_mode'] == "poll"
233 
+    end
234 
+
235 
+    def event_for_hit(hit_id)
236 
+      if memory['hits'][hit_id].is_a?(Hash)
237 
+        Event.find_by_id(memory['hits'][hit_id]['event_id'])
238 
+      else
239 
+        nil
240 
+      end
241 
+    end
242 
+
243 
+    def hit_type(hit_id)
244 
+      if memory['hits'][hit_id].is_a?(Hash) && memory['hits'][hit_id]['type']
245 
+        memory['hits'][hit_id]['type']
246 
+      else
247 
+        'user'
248 
+      end
249 
+    end
250 
+
169 251 
     def review_hits
170 252 
       reviewable_hit_ids = RTurk::GetReviewableHITs.create.hit_ids
171 253 
       my_reviewed_hit_ids = reviewable_hit_ids & (memory['hits'] || {}).keys
172 254 
       if reviewable_hit_ids.length > 0
173 255 
         log "MTurk reports #{reviewable_hit_ids.length} HITs, of which I own [#{my_reviewed_hit_ids.to_sentence}]"
174 256 
       end
257 
+
175 258 
       my_reviewed_hit_ids.each do |hit_id|
176 259 
         hit = RTurk::Hit.new(hit_id)
177 260 
         assignments = hit.assignments
178 261
179 262 
         log "Looking at HIT #{hit_id}.  I found #{assignments.length} assignments#{" with the statuses: #{assignments.map(&:status).to_sentence}" if assignments.length > 0}"
180 263 
         if assignments.length == hit.max_assignments && assignments.all? { |assignment| assignment.status == "Submitted" }
181 
-          payload = { 'answers' => assignments.map(&:answers) }
182 
-
183 
-          if options['take_majority'] == "true"
184 
-            counts = {}
185 
-            options['hit']['questions'].each do |question|
186 
-              question_counts = question['selections'].inject({}) { |memo, selection| memo[selection['key']] = 0; memo }
187 
-              assignments.each do |assignment|
188 
-                answers = ActiveSupport::HashWithIndifferentAccess.new(assignment.answers)
189 
-                answer = answers[question['key']]
190 
-                question_counts[answer] += 1
264 
+          inbound_event = event_for_hit(hit_id)
265 
+
266 
+          if hit_type(hit_id) == 'poll'
267 
+            # handle completed polls
268 
+
269 
+            log "Handling a poll: #{hit_id}"
270 
+
271 
+            scores = {}
272 
+            assignments.each do |assignment|
273 
+              assignment.answers.each do |index, rating|
274 
+                scores[index] ||= 0
275 
+                scores[index] += rating.to_i
191 276 
               end
192 
-              counts[question['key']] = question_counts
193 277 
             end
194 
-            payload['counts'] = counts
195 278
196 
-            majority_answer = counts.inject({}) do |memo, (key, question_counts)|
197 
-              memo[key] = question_counts.to_a.sort {|a, b| a.last <=> b.last }.last.first
198 
-              memo
199 
-            end
200 
-            payload['majority_answer'] = majority_answer
201 
-
202 
-            if all_questions_are_numeric?
203 
-              average_answer = counts.inject({}) do |memo, (key, question_counts)|
204 
-                sum = divisor = 0
205 
-                question_counts.to_a.each do |num, count|
206 
-                  sum += num.to_s.to_f * count
207 
-                  divisor += count
279 
+            top_answer = scores.to_a.sort {|b, a| a.last <=> b.last }.first.first
280 
+
281 
+            payload = {
282 
+              'answers' => memory['hits'][hit_id]['answers'],
283 
+              'poll' => assignments.map(&:answers),
284 
+              'best_answer' => memory['hits'][hit_id]['answers'][top_answer.to_i - 1]
285 
+            }
286 
+
287 
+            event = create_event :payload => payload
288 
+            log "Event emitted with answer(s) for poll", :outbound_event => event, :inbound_event => inbound_event
289 
+          else
290 
+            # handle normal completed HITs
291 
+            payload = { 'answers' => assignments.map(&:answers) }
292 
+
293 
+            if take_majority?
294 
+              counts = {}
295 
+              options['hit']['questions'].each do |question|
296 
+                question_counts = question['selections'].inject({}) { |memo, selection| memo[selection['key']] = 0; memo }
297 
+                assignments.each do |assignment|
298 
+                  answers = ActiveSupport::HashWithIndifferentAccess.new(assignment.answers)
299 
+                  answer = answers[question['key']]
300 
+                  question_counts[answer] += 1
208 301 
                 end
209 
-                memo[key] = sum / divisor.to_f
302 
+                counts[question['key']] = question_counts
303 
+              end
304 
+              payload['counts'] = counts
305 
+
306 
+              majority_answer = counts.inject({}) do |memo, (key, question_counts)|
307 
+                memo[key] = question_counts.to_a.sort {|a, b| a.last <=> b.last }.last.first
210 308 
                 memo
211 309 
               end
212 
-              payload['average_answer'] = average_answer
310 
+              payload['majority_answer'] = majority_answer
311 
+
312 
+              if all_questions_are_numeric?
313 
+                average_answer = counts.inject({}) do |memo, (key, question_counts)|
314 
+                  sum = divisor = 0
315 
+                  question_counts.to_a.each do |num, count|
316 
+                    sum += num.to_s.to_f * count
317 
+                    divisor += count
318 
+                  end
319 
+                  memo[key] = sum / divisor.to_f
320 
+                  memo
321 
+                end
322 
+                payload['average_answer'] = average_answer
323 
+              end
213 324 
             end
214 
-          end
215 325
216 
-          event = create_event :payload => payload
217 
-          log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => Event.find_by_id(memory['hits'][hit_id])
326 
+            if create_poll?
327 
+              questions = []
328 
+              selections = 5.times.map { |i| { 'key' => i+1, 'text' => i+1 } }.reverse
329 
+              assignments.length.times do |index|
330 
+                questions << {
331 
+                  'type' => "selection",
332 
+                  'name' => "Item #{index + 1}",
333 
+                  'key' => index,
334 
+                  'required' => "true",
335 
+                  'question' => Utils.interpolate_jsonpaths(options['poll_options']['row_template'], assignments[index].answers),
336 
+                  'selections' => selections
337 
+                }
338 
+              end
339 
+
340 
+              poll_hit = create_hit 'title' => options['poll_options']['title'],
341 
+                                    'description' => options['poll_options']['instructions'],
342 
+                                    'questions' => questions,
343 
+                                    'assignments' => options['poll_options']['assignments'],
344 
+                                    'lifetime_in_seconds' => options['poll_options']['lifetime_in_seconds'],
345 
+                                    'reward' => options['poll_options']['reward'],
346 
+                                    'payload' => inbound_event && inbound_event.payload,
347 
+                                    'metadata' => { 'type' => 'poll',
348 
+                                                    'original_hit' => hit_id,
349 
+                                                    'answers' => assignments.map(&:answers),
350 
+                                                    'event_id' => inbound_event && inbound_event.id }
351 
+
352 
+              log "Poll HIT created with ID #{poll_hit.id} and URL #{poll_hit.url}.  Original HIT: #{hit_id}", :inbound_event => inbound_event
353 
+            else
354 
+              event = create_event :payload => payload
355 
+              log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => inbound_event
356 
+            end
357 
+          end
218 358
219 359 
           assignments.each(&:approve!)
220 360 
           hit.dispose!
 
@@ -232,22 +372,35 @@ module Agents
232 372 
       end
233 373 
     end
234 374
235 
-    def create_hit(event = nil)
236 
-      payload = event ? event.payload : {}
237 
-      title = Utils.interpolate_jsonpaths(options['hit']['title'], payload).strip
238 
-      description = Utils.interpolate_jsonpaths(options['hit']['description'], payload).strip
239 
-      questions = Utils.recursively_interpolate_jsonpaths(options['hit']['questions'], payload)
375 
+    def create_basic_hit(event = nil)
376 
+      hit = create_hit 'title' => options['hit']['title'],
377 
+                       'description' => options['hit']['description'],
378 
+                       'questions' => options['hit']['questions'],
379 
+                       'assignments' => options['hit']['assignments'],
380 
+                       'lifetime_in_seconds' => options['hit']['lifetime_in_seconds'],
381 
+                       'reward' => options['hit']['reward'],
382 
+                       'payload' => event && event.payload,
383 
+                       'metadata' => { 'event_id' => event && event.id }
384 
+
385 
+      log "HIT created with ID #{hit.id} and URL #{hit.url}", :inbound_event => event
386 
+    end
387 
+
388 
+    def create_hit(opts = {})
389 
+      payload = opts['payload'] || {}
390 
+      title = Utils.interpolate_jsonpaths(opts['title'], payload).strip
391 
+      description = Utils.interpolate_jsonpaths(opts['description'], payload).strip
392 
+      questions = Utils.recursively_interpolate_jsonpaths(opts['questions'], payload)
240 393 
       hit = RTurk::Hit.create(:title => title) do |hit|
241 
-        hit.max_assignments = (options['hit']['assignments'] || 1).to_i
394 
+        hit.max_assignments = (opts['assignments'] || 1).to_i
242 395 
         hit.description = description
243 
-        hit.lifetime = (options['hit']['lifetime_in_seconds'] || 24 * 60 * 60).to_i
396 
+        hit.lifetime = (opts['lifetime_in_seconds'] || 24 * 60 * 60).to_i
244 397 
         hit.question_form AgentQuestionForm.new(:title => title, :description => description, :questions => questions)
245 
-        hit.reward = (options['hit']['reward'] || 0.05).to_f
398 
+        hit.reward = (opts['reward'] || 0.05).to_f
246 399 
         #hit.qualifications.add :approval_rate, { :gt => 80 }
247 400 
       end
248 401 
       memory['hits'] ||= {}
249 
-      memory['hits'][hit.id] = event && event.id
250 
-      log "HIT created with ID #{hit.id} and URL #{hit.url}", :inbound_event => event
402 
+      memory['hits'][hit.id] = opts['metadata'] || {}
403 
+      hit
251 404 
     end
252 405
253 406 
     # RTurk Question Form
 
@@ -328,4 +481,4 @@ module Agents
328 481 
       end
329 482 
     end
330 483 
   end
331 
-end
484 
+end

+ 62 - 0
app/models/agents/webhook_agent.rb
Parādīt failu 

@@ -0,0 +1,62 @@
1 
+module Agents
2 
+  class WebhookAgent < Agent
3 
+    cannot_be_scheduled!
4 
+
5 
+    description  do
6 
+        <<-MD
7 
+        Use this Agent to create events by receiving webhooks from any source.
8 
+
9 
+        In order to create events with this agent, make a POST request to:
10 
+        ```
11 
+           https://#{ENV['DOMAIN']}/users/#{user.id}/webhooks/#{id || '<id>'}/:secret
12 
+        ``` where `:secret` is specified in your options.
13 
+
14 
+        The
15 
+
16 
+        Options:
17 
+
18 
+          * `secret` - A token that the host will provide for authentication.
19 
+          * `expected_receive_period_in_days` - How often you expect to receive
20 
+            events this way. Used to determine if the agent is working.
21 
+          * `payload_path` - JSONPath of the attribute in the POST body to be
22 
+            used as the Event payload.
23 
+      MD
24 
+    end
25 
+
26 
+    event_description do
27 
+      <<-MD
28 
+        The event payload is base on the value of the `payload_path` option,
29 
+        which is set to `#{options['payload_path']}`.
30 
+      MD
31 
+    end
32 
+
33 
+    def default_options
34 
+      { "secret" => "supersecretstring",
35 
+        "expected_receive_period_in_days" => 1,
36 
+        "payload_path" => "payload"}
37 
+    end
38 
+
39 
+    def receive_webhook(params)
40 
+      secret = params.delete('secret')
41 
+      return ["Not Authorized", 401] unless secret == options['secret']
42 
+
43 
+      create_event(:payload => payload_for(params))
44 
+
45 
+      ['Event Created', 201]
46 
+    end
47 
+
48 
+    def working?
49 
+      event_created_within(options['expected_receive_period_in_days']) && !recent_error_logs?
50 
+    end
51 
+
52 
+    def validate_options
53 
+      unless options['secret'].present?
54 
+        errors.add(:base, "Must specify a secret for 'Authenticating' requests")
55 
+      end
56 
+    end
57 
+
58 
+    def payload_for(params)
59 
+      Utils.value_at(params, options['payload_path']) || {}
60 
+    end
61 
+  end
62 
+end

+ 30 - 1
lib/capistrano/sync.rb
Parādīt failu 

@@ -1,5 +1,6 @@
1 1 
 require 'yaml'
2 2 
 require 'pathname'
3 
+require 'dotenv'
3 4
4 5 
 # Edited by Andrew Cantino.  Based on: https://gist.github.com/339471
5 6 
 
@@ -99,6 +100,28 @@ namespace :sync do
99 100 
     return database["#{db}"]['username'], database["#{db}"]['password'], database["#{db}"]['database'], database["#{db}"]['host']
100 101 
   end
101 102
103 
+  # Used by remote_database_config to parse the remote .env file.  Depends on the dotenv-rails gem.
104 
+  class RemoteEnvLoader < Dotenv::Environment
105 
+    def initialize(data)
106 
+      @data = data
107 
+      load
108 
+    end
109 
+
110 
+    def with_loaded_env
111 
+      begin
112 
+        saved_env = ENV.to_hash.dup
113 
+        ENV.update(self)
114 
+        yield
115 
+      ensure
116 
+        ENV.replace(saved_env)
117 
+      end
118 
+    end
119 
+
120 
+    def read
121 
+      @data.split("\n")
122 
+    end
123 
+  end
124 
+
102 125 
   #
103 126 
   # Reads the database credentials from the remote config/database.yml file
104 127 
   # +db+ the name of the environment to get the credentials for
 
@@ -106,7 +129,13 @@ namespace :sync do
106 129 
   #
107 130 
   def remote_database_config(db)
108 131 
     remote_config = capture("cat #{current_path}/config/database.yml")
109 
-    database = YAML::load(remote_config)
132 
+    remote_env = capture("cat #{current_path}/.env")
133 
+
134 
+    database = nil
135 
+    RemoteEnvLoader.new(remote_env).with_loaded_env do
136 
+      database = YAML::load(ERB.new(remote_config).result)
137 
+    end
138 
+
110 139 
     return database["#{db}"]['username'], database["#{db}"]['password'], database["#{db}"]['database'], database["#{db}"]['host']
111 140 
   end
112 141

+ 172 - 19
spec/models/agents/human_task_agent_spec.rb
Parādīt failu 

@@ -108,7 +108,43 @@ describe Agents::HumanTaskAgent do
108 108 
       @checker.should_not be_valid
109 109 
     end
110 110
111 
-    it "requires that all questions be of type 'selection' when `take_majority` is `true`" do
111 
+    it "requires that 'poll_options' be present and populated when 'combination_mode' is set to 'poll'" do
112 
+      @checker.options['combination_mode'] = "poll"
113 
+      @checker.should_not be_valid
114 
+      @checker.options['poll_options'] = {}
115 
+      @checker.should_not be_valid
116 
+      @checker.options['poll_options'] = { 'title' => "Take a poll about jokes",
117 
+                                           'instructions' => "Rank these by how funny they are",
118 
+                                           'assignments' => 3,
119 
+                                           'row_template' => "<$.joke>" }
120 
+      @checker.should be_valid
121 
+      @checker.options['poll_options'] = { 'instructions' => "Rank these by how funny they are",
122 
+                                           'assignments' => 3,
123 
+                                           'row_template' => "<$.joke>" }
124 
+      @checker.should_not be_valid
125 
+      @checker.options['poll_options'] = { 'title' => "Take a poll about jokes",
126 
+                                           'assignments' => 3,
127 
+                                           'row_template' => "<$.joke>" }
128 
+      @checker.should_not be_valid
129 
+      @checker.options['poll_options'] = { 'title' => "Take a poll about jokes",
130 
+                                           'instructions' => "Rank these by how funny they are",
131 
+                                           'row_template' => "<$.joke>" }
132 
+      @checker.should_not be_valid
133 
+      @checker.options['poll_options'] = { 'title' => "Take a poll about jokes",
134 
+                                           'instructions' => "Rank these by how funny they are",
135 
+                                           'assignments' => 3}
136 
+      @checker.should_not be_valid
137 
+    end
138 
+
139 
+    it "requires that all questions be of type 'selection' when 'combination_mode' is 'take_majority'" do
140 
+      @checker.options['combination_mode'] = "take_majority"
141 
+      @checker.should_not be_valid
142 
+      @checker.options['hit']['questions'][1]['type'] = "selection"
143 
+      @checker.options['hit']['questions'][1]['selections'] = @checker.options['hit']['questions'][0]['selections']
144 
+      @checker.should be_valid
145 
+    end
146 
+
147 
+    it "accepts 'take_majority': 'true' for legacy support" do
112 148 
       @checker.options['take_majority'] = "true"
113 149 
       @checker.should_not be_valid
114 150 
       @checker.options['hit']['questions'][1]['type'] = "selection"
 
@@ -126,7 +162,7 @@ describe Agents::HumanTaskAgent do
126 162
127 163 
     it "should check for reviewable HITs frequently" do
128 164 
       mock(@checker).review_hits.twice
129 
-      mock(@checker).create_hit.once
165 
+      mock(@checker).create_basic_hit.once
130 166 
       @checker.check
131 167 
       @checker.check
132 168 
     end
 
@@ -135,7 +171,7 @@ describe Agents::HumanTaskAgent do
135 171 
       now = Time.now
136 172 
       stub(Time).now { now }
137 173 
       mock(@checker).review_hits.times(3)
138 
-      mock(@checker).create_hit.twice
174 
+      mock(@checker).create_basic_hit.twice
139 175 
       @checker.check
140 176 
       now += 1 * 60 * 60
141 177 
       @checker.check
 
@@ -144,7 +180,7 @@ describe Agents::HumanTaskAgent do
144 180 
     end
145 181
146 182 
     it "should ignore events" do
147 
-      mock(@checker).create_hit(anything).times(0)
183 
+      mock(@checker).create_basic_hit(anything).times(0)
148 184 
       @checker.receive([events(:bob_website_agent_event)])
149 185 
     end
150 186 
   end
 
@@ -155,7 +191,7 @@ describe Agents::HumanTaskAgent do
155 191 
       now = Time.now
156 192 
       stub(Time).now { now }
157 193 
       mock(@checker).review_hits.times(3)
158 
-      mock(@checker).create_hit.times(0)
194 
+      mock(@checker).create_basic_hit.times(0)
159 195 
       @checker.check
160 196 
       now += 1 * 60 * 60
161 197 
       @checker.check
 
@@ -164,7 +200,7 @@ describe Agents::HumanTaskAgent do
164 200 
     end
165 201
166 202 
     it "should create HITs based on events" do
167 
-      mock(@checker).create_hit(events(:bob_website_agent_event)).times(1)
203 
+      mock(@checker).create_basic_hit(events(:bob_website_agent_event)).times(1)
168 204 
       @checker.receive([events(:bob_website_agent_event)])
169 205 
     end
170 206 
   end
 
@@ -181,7 +217,7 @@ describe Agents::HumanTaskAgent do
181 217 
       mock(hitInterface).question_form(instance_of Agents::HumanTaskAgent::AgentQuestionForm) { |agent_question_form_instance| question_form = agent_question_form_instance }
182 218 
       mock(RTurk::Hit).create(:title => "Hi Joe").yields(hitInterface) { hitInterface }
183 219
184 
-      @checker.send :create_hit, @event
220 
+      @checker.send :create_basic_hit, @event
185 221
186 222 
       hitInterface.max_assignments.should == @checker.options['hit']['assignments']
187 223 
       hitInterface.reward.should == @checker.options['hit']['reward']
 
@@ -192,7 +228,7 @@ describe Agents::HumanTaskAgent do
192 228 
       xml.should include("<Text>Make something for Joe</Text>")
193 229 
       xml.should include("<DisplayName>Joe Question 1</DisplayName>")
194 230
195 
-      @checker.memory['hits'][123].should == @event.id
231 
+      @checker.memory['hits'][123]['event_id'].should == @event.id
196 232 
     end
197 233
198 234 
     it "works without an event too" do
 
@@ -201,7 +237,7 @@ describe Agents::HumanTaskAgent do
201 237 
       hitInterface.id = 123
202 238 
       mock(hitInterface).question_form(instance_of Agents::HumanTaskAgent::AgentQuestionForm)
203 239 
       mock(RTurk::Hit).create(:title => "Hi").yields(hitInterface) { hitInterface }
204 
-      @checker.send :create_hit
240 
+      @checker.send :create_basic_hit
205 241 
       hitInterface.max_assignments.should == @checker.options['hit']['assignments']
206 242 
       hitInterface.reward.should == @checker.options['hit']['reward']
207 243 
     end
 
@@ -259,8 +295,8 @@ describe Agents::HumanTaskAgent do
259 295
260 296 
       # It knows about two HITs from two different events.
261 297 
       @checker.memory['hits'] = {}
262 
-      @checker.memory['hits']["JH3132836336DHG"] = @event.id
263 
-      @checker.memory['hits']["JH39AA63836DHG"] = event2.id
298 
+      @checker.memory['hits']["JH3132836336DHG"] = { 'event_id' => @event.id }
299 
+      @checker.memory['hits']["JH39AA63836DHG"] = { 'event_id' => event2.id }
264 300
265 301 
       hit_ids = %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345]
266 302 
       mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { hit_ids } } # It sees 3 HITs.
 
@@ -273,7 +309,7 @@ describe Agents::HumanTaskAgent do
273 309 
     end
274 310
275 311 
     it "shouldn't do anything if an assignment isn't ready" do
276 
-      @checker.memory['hits'] = { "JH3132836336DHG" => @event.id }
312 
+      @checker.memory['hits'] = { "JH3132836336DHG" => { 'event_id' => @event.id } }
277 313 
       mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
278 314 
       assignments = [
279 315 
         FakeAssignment.new(:status => "Accepted", :answers => {}),
 
@@ -288,11 +324,11 @@ describe Agents::HumanTaskAgent do
288 324 
       @checker.send :review_hits
289 325
290 326 
       assignments.all? {|a| a.approved == true }.should be_false
291 
-      @checker.memory['hits'].should == { "JH3132836336DHG" => @event.id }
327 
+      @checker.memory['hits'].should == { "JH3132836336DHG" => { 'event_id' => @event.id } }
292 328 
     end
293 329
294 330 
     it "shouldn't do anything if an assignment is missing" do
295 
-      @checker.memory['hits'] = { "JH3132836336DHG" => @event.id }
331 
+      @checker.memory['hits'] = { "JH3132836336DHG" => { 'event_id' => @event.id } }
296 332 
       mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
297 333 
       assignments = [
298 334 
         FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "feedback"=>"Take 2"})
 
@@ -306,11 +342,11 @@ describe Agents::HumanTaskAgent do
306 342 
       @checker.send :review_hits
307 343
308 344 
       assignments.all? {|a| a.approved == true }.should be_false
309 
-      @checker.memory['hits'].should == { "JH3132836336DHG" => @event.id }
345 
+      @checker.memory['hits'].should == { "JH3132836336DHG" => { 'event_id' => @event.id } }
310 346 
     end
311 347
312 348 
     it "should create events when all assignments are ready" do
313 
-      @checker.memory['hits'] = { "JH3132836336DHG" => @event.id }
349 
+      @checker.memory['hits'] = { "JH3132836336DHG" => { 'event_id' => @event.id } }
314 350 
       mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
315 351 
       assignments = [
316 352 
         FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"neutral", "feedback"=>""}),
 
@@ -337,8 +373,8 @@ describe Agents::HumanTaskAgent do
337 373
338 374 
     describe "taking majority votes" do
339 375 
       before do
340 
-        @checker.options['take_majority'] = "true"
341 
-        @checker.memory['hits'] = { "JH3132836336DHG" => @event.id }
376 
+        @checker.options['combination_mode'] = "take_majority"
377 
+        @checker.memory['hits'] = { "JH3132836336DHG" => { 'event_id' => @event.id } }
342 378 
         mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
343 379 
       end
344 380 
 
@@ -386,6 +422,10 @@ describe Agents::HumanTaskAgent do
386 422 
       end
387 423
388 424 
       it "should also provide an average answer when all questions are numeric" do
425 
+        # it should accept 'take_majority': 'true' as well for legacy support.  Demonstrating that here.
426 
+        @checker.options.delete :combination_mode
427 
+        @checker.options['take_majority'] = "true"
428 
+
389 429 
         @checker.options['hit']['questions'] = [
390 430 
           {
391 431 
             'type' => "selection",
 
@@ -435,5 +475,118 @@ describe Agents::HumanTaskAgent do
435 475 
         @checker.memory['hits'].should == {}
436 476 
       end
437 477 
     end
478 
+
479 
+    describe "creating and reviewing polls" do
480 
+      before do
481 
+        @checker.options['combination_mode'] = "poll"
482 
+        @checker.options['poll_options'] = {
483 
+          'title' => "Hi!",
484 
+          'instructions' => "hello!",
485 
+          'assignments' => 2,
486 
+          'row_template' => "This is <.sentiment>"
487 
+        }
488 
+        @event.save!
489 
+        mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
490 
+      end
491 
+
492 
+      it "creates a poll using the row_template, message, and correct number of assignments" do
493 
+        @checker.memory['hits'] = { "JH3132836336DHG" => { 'event_id' => @event.id } }
494 
+
495 
+        # Mock out the HIT's submitted assignments.
496 
+        assignments = [
497 
+          FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"sad",     "feedback"=>"This is my feedback 1"}),
498 
+          FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"neutral", "feedback"=>"This is my feedback 2"}),
499 
+          FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy",   "feedback"=>"This is my feedback 3"}),
500 
+          FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy",   "feedback"=>"This is my feedback 4"})
501 
+        ]
502 
+        hit = FakeHit.new(:max_assignments => 4, :assignments => assignments)
503 
+        mock(RTurk::Hit).new("JH3132836336DHG") { hit }
504 
+
505 
+        @checker.memory['hits']["JH3132836336DHG"].should be_present
506 
+
507 
+        # Setup mocks for HIT creation
508 
+
509 
+        question_form = nil
510 
+        hitInterface = OpenStruct.new
511 
+        hitInterface.id = "JH39AA63836DH12345"
512 
+        mock(hitInterface).question_form(instance_of Agents::HumanTaskAgent::AgentQuestionForm) { |agent_question_form_instance| question_form = agent_question_form_instance }
513 
+        mock(RTurk::Hit).create(:title => "Hi!").yields(hitInterface) { hitInterface }
514 
+
515 
+        # And finally, the test.
516 
+
517 
+        lambda {
518 
+          @checker.send :review_hits
519 
+        }.should change { Event.count }.by(0) # it does not emit an event until all poll results are in
520 
+
521 
+        # it approves the existing assignments
522 
+
523 
+        assignments.all? {|a| a.approved == true }.should be_true
524 
+        hit.should be_disposed
525 
+
526 
+        # it creates a new HIT for the poll
527 
+
528 
+        hitInterface.max_assignments.should == @checker.options['poll_options']['assignments']
529 
+        hitInterface.description.should == @checker.options['poll_options']['instructions']
530 
+
531 
+        xml = question_form.to_xml
532 
+        xml.should include("<Text>This is happy</Text>")
533 
+        xml.should include("<Text>This is neutral</Text>")
534 
+        xml.should include("<Text>This is sad</Text>")
535 
+
536 
+        @checker.save
537 
+        @checker.reload
538 
+        @checker.memory['hits']["JH3132836336DHG"].should_not be_present
539 
+        @checker.memory['hits']["JH39AA63836DH12345"].should be_present
540 
+        @checker.memory['hits']["JH39AA63836DH12345"]['event_id'].should == @event.id
541 
+        @checker.memory['hits']["JH39AA63836DH12345"]['type'].should == "poll"
542 
+        @checker.memory['hits']["JH39AA63836DH12345"]['original_hit'].should == "JH3132836336DHG"
543 
+        @checker.memory['hits']["JH39AA63836DH12345"]['answers'].length.should == 4
544 
+      end
545 
+
546 
+      it "emits an event when all poll results are in, containing the data from the best answer, plus all others" do
547 
+        original_answers = [
548 
+          { 'sentiment' => "sad",     'feedback' => "This is my feedback 1"},
549 
+          { 'sentiment' => "neutral", 'feedback' => "This is my feedback 2"},
550 
+          { 'sentiment' => "happy",   'feedback' => "This is my feedback 3"},
551 
+          { 'sentiment' => "happy",   'feedback' => "This is my feedback 4"}
552 
+        ]
553 
+
554 
+        @checker.memory['hits'] = {
555 
+          'JH39AA63836DH12345' => {
556 
+            'type' => 'poll',
557 
+            'original_hit' => "JH3132836336DHG",
558 
+            'answers' => original_answers,
559 
+            'event_id' => 345
560 
+          }
561 
+        }
562 
+
563 
+        # Mock out the HIT's submitted assignments.
564 
+        assignments = [
565 
+          FakeAssignment.new(:status => "Submitted", :answers => {"1" => "2", "2" => "5", "3" => "3", "4" => "2"}),
566 
+          FakeAssignment.new(:status => "Submitted", :answers => {"1" => "3", "2" => "4", "3" => "1", "4" => "4"})
567 
+        ]
568 
+        hit = FakeHit.new(:max_assignments => 2, :assignments => assignments)
569 
+        mock(RTurk::Hit).new("JH39AA63836DH12345") { hit }
570 
+
571 
+        @checker.memory['hits']["JH39AA63836DH12345"].should be_present
572 
+
573 
+        lambda {
574 
+          @checker.send :review_hits
575 
+        }.should change { Event.count }.by(1)
576 
+
577 
+        # It emits an event
578 
+
579 
+        @checker.events.last.payload['answers'].should == original_answers
580 
+        @checker.events.last.payload['poll'].should == [{"1" => "2", "2" => "5", "3" => "3", "4" => "2"}, {"1" => "3", "2" => "4", "3" => "1", "4" => "4"}]
581 
+        @checker.events.last.payload['best_answer'].should == {'sentiment' => "neutral", 'feedback' => "This is my feedback 2"}
582 
+
583 
+        # it approves the existing assignments
584 
+
585 
+        assignments.all? {|a| a.approved == true }.should be_true
586 
+        hit.should be_disposed
587 
+
588 
+        @checker.memory['hits'].should be_empty
589 
+      end
590 
+    end
438 591 
   end
439 
-end
592 
+end

+ 31 - 0
spec/models/agents/webhook_agent_spec.rb
Parādīt failu 

@@ -0,0 +1,31 @@
1 
+require 'spec_helper'
2 
+
3 
+describe Agents::WebhookAgent do
4 
+  let(:agent) do
5 
+    _agent = Agents::WebhookAgent.new(:name => 'webhook',
6 
+                                      :options => { 'secret' => 'foobar', 'payload_path' => 'payload' })
7 
+    _agent.user = users(:bob)
8 
+    _agent.save!
9 
+    _agent
10 
+  end
11 
+  let(:payload) { {'some' => 'info'} }
12 
+
13 
+  describe 'receive_webhook' do
14 
+    it 'should create event if secret matches' do
15 
+      out = nil
16 
+      lambda {
17 
+        out = agent.receive_webhook('secret' => 'foobar', 'payload' => payload)
18 
+      }.should change { Event.count }.by(1)
19 
+      out.should eq(['Event Created', 201])
20 
+      Event.last.payload.should eq(payload)
21 
+    end
22 
+
23 
+    it 'should not create event if secrets dont match' do
24 
+      out = nil
25 
+      lambda {
26 
+        out = agent.receive_webhook('secret' => 'bazbat', 'payload' => payload)
27 
+      }.should change { Event.count }.by(0)
28 
+      out.should eq(['Not Authorized', 401])
29 
+    end
30 
+  end
31 
+end